#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Created by master on 2017/3/26

import bs4
from bs4 import BeautifulSoup
import os
import zlib
import codecs
import time
from urllib import request


class Spider(object):
    @staticmethod
    def get_html(url):
        req = request.Request(url)
        req.add_header('User-Agent',
                       'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 UBrowser/5.5.9703.2 Safari/537.36')
        req.add_header('Accept-encoding', 'gzip')
        with request.urlopen(req) as response:
            gzipd = response.headers.get('Content-Encoding')
            if gzipd == 'gzip':
                data = zlib.decompress(response.read(), 16 + zlib.MAX_WBITS)

            else:
                data = response.read()
            return BeautifulSoup(data.decode("utf-8"), "lxml")


if __name__ == '__main__':
    html = Spider.get_html("https://www.jd.com/?cu=true&utm_source=infinitynewtab.com&utm_medium=tuiguang&utm_campaign=t_45363_&utm_term=e7663eb36b9346e48b8934e38bc26546-p_1&abt=3")

    # print(html.find_all("ul",class_="m-cvrlst f-cb"))
    # print(html.find_all(id="m-pl-container"))
    print(html)
